#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# @Time    : 2018/11/15 19:45
# @Author  : liujiantao
# @Site    : 
# @File    : op_data_analysis.py
# @Software: PyCharm

from tiancheng.base.base_helper import *
op_rd1 = get_operation_round1_new()
op_tr = get_operation_train_new()
tag = get_tag_train_new()
# 测试集没有tag
op_rd1[tag_hd.Tag] = -1
op_tr = pd.merge(left=op_tr,right=tag, on=tag_hd.UID)
cols = operation_header + [tag_hd.Tag]
op_merge = pd.concat([op_tr[cols],op_rd1[cols]])
# 1)数据缺失率
 #按照行来数非缺失值的个数
op_merge[op_hd.data_effective_num] = list(op_merge.count(axis=1))
# 每行有效值的个数/总数
op_merge[op_hd.data_lose_rate] = list((op_merge.shape[1] - op_merge.count(axis=1))/op_merge.shape[1])
# 交易日期处理
# 独热编码

# 1)数据异常率

## 数据预处理
# 1）异常值处理、异常点检测
# 操作日期
op_merge[op_hd.time] = op_merge[op_hd.time].map(lambda x:x[:2])
# 操作类型
op_merge = filed2encode_f(op_merge, op_hd.mode)
# 2）缺失数据补均值 , 纵向用缺失值上面的值替换缺失值
op_merge.fillna(axis=1,method='ffill')
op_merge = filed2encode_f(op_merge, op_hd.version)
# op_merge[op_hd.version].fillna(axis=0,method='ffill')
op_merge[op_hd.version].fillna('7.0.9')
op_merge = filed2encode_f(op_merge, op_hd.version)
# 操作设备参数1
op_merge = filed2encode_f(op_merge, op_hd.device1)
op_merge = filed2encode_f(op_merge, op_hd.device2)
# 设备操作唯一标识
# op_merge.groupby(op_hd.device_code1)[op_hd.UID].count().reset_index()
op_merge = filed2encode_f(op_merge, op_hd.device_code1)
op_merge = filed2encode_f(op_merge, op_hd.device_code2)
op_merge = filed2encode_f(op_merge, op_hd.device_code3)
# mac地址
op_merge = filed2encode_f(op_merge, op_hd.mac1)
op_merge = filed2encode_f(op_merge, op_hd.mac2)
op_merge.groupby(op_hd.mac1)[op_hd.UID].count().reset_index()
# IP 地址
# op_merge.groupby(op_hd.ip1_sub)[op_hd.UID].count().reset_index()
op_merge = filed2encode_f(op_merge, op_hd.ip1)
op_merge = filed2encode_f(op_merge, op_hd.ip2)
op_merge = filed2encode_f(op_merge, op_hd.ip1_sub)
op_merge = filed2encode_f(op_merge, op_hd.ip2_sub)
# wifi
op_merge = filed2encode_f(op_merge, op_hd.ip2_sub)
# 客户端版本号
# op_merge.groupby(op_hd.wifi)[op_hd.UID].count().reset_index()
op_merge = filed2encode_f(op_merge, op_hd.wifi)
# 地理位置
op_merge = filed2encode_f(op_merge, op_hd.geo_code)
op_merge.groupby(op_hd.geo_code)[op_hd.UID].count().reset_index()
# 前向填充，使用默认是上一行的值,设置axis=1可以使用列进行填充
op_merge = op_merge.fillna(method="ffill")
# # 后向填充，使用下一行的值,不存在的时候就不填充
op_merge = op_merge.fillna(method="bfill")
op_merge.fillna(-1, inplace=True)  # 全空默认 -1
# 3）去重
# op_merge[op_hd.UID=='10035'][operation_header]
pd.to_pickle(op_merge,features_base_path+"op_merge01.pkl")

# op_merge.head(1)
# op_merge.groupby(op_hd.mode).count().reset_index()
# op_day = op_merge[[tag_hd.UID,op_hd.day,tag_hd.Tag]].groupby(op_hd.day).res
# op_day.count()
# print("stage 1")

# tr_rd1 = bh.get_transaction_round1_new()
